package wvtool;

import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.io.CharSink;
import com.google.common.io.CharSource;
import com.google.common.io.Files;
import edu.udo.cs.wvtool.config.WVTConfiguration;
import edu.udo.cs.wvtool.config.WVTConfigurationFact;
import edu.udo.cs.wvtool.generic.tokenizer.SimpleTokenizer;
import edu.udo.cs.wvtool.generic.wordfilter.AbstractStopWordFilter;
import edu.udo.cs.wvtool.main.WVTDocumentInfo;
import edu.udo.cs.wvtool.main.WVTFileInputList;
import edu.udo.cs.wvtool.util.WVToolException;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;

/**
 * 作者: LDL
 * 功能说明:
 * 创建日期: 2015/6/16 17:48
 */
public class WVToolExample {
    public static void main(String[] args) throws WVToolException, IOException {

        createVector("F:/guideweibo/weibotrans");
        createLibSVM("F:/guideweibo/weibotrans");

    }

    public static void createVector(String path) throws IOException, WVToolException {
        WVTool wvt = new WVTool(false);
        WVTConfiguration config = new WVTConfiguration();

        config.setConfigurationRule(WVTConfiguration.STEP_TOKENIZER,new WVTConfigurationFact(new ChineseTokenizer(new SimpleTokenizer())));
        config.setConfigurationRule(WVTConfiguration.STEP_WORDFILTER,new WVTConfigurationFact(new AbstractStopWordFilter(2) {
            @Override
            protected boolean isStopword(String s) {
                return false;
            }
        }));
        // Initialize the input list with two classes
        WVTFileInputList list = new WVTFileInputList(2);

        File dir1 = new File(path+"/0");
        File dir2 = new File(path+"/1");
        File[] files1 = dir1.listFiles();
        File[] files2 = dir2.listFiles();

        int count = 0;
        for(File file : files1){
            if(count>=2500){
                break;
            }
            list.addEntry(new WVTDocumentInfo(file.getAbsolutePath(), "txt", "", "chinese", 0));
            count++;
        }

        for(File file : files2){
            list.addEntry(new WVTDocumentInfo(file.getAbsolutePath(), "txt", "", "chinese", 1));
        }
        // Generate the word list

        WVTWordList wordList = wvt.createWordList(list, config);

        Comparator<WVTWord> comparator = new Comparator<WVTWord>() {
            @Override
            public int compare(WVTWord o1, WVTWord o2) {
                return o2.getDocumentFrequency()-o1.getDocumentFrequency();
            }
        };
        wordList.getWordList().sort(comparator);
        wordList.setWordList(wordList.getWordList().subList(0,1000));

        //wordList.pruneByFrequency(0, 5);

        wordList.storePlain(new FileWriter(path+"/wordlist.txt"));

        // Create the word vectors
        wordList.store(new FileWriter(path + "/wordVector.txt"));
        // Set up an output filter (write sparse vectors to a file)
        FileWriter outFile = new FileWriter(path+"/wv.txt");
        WordVectorWriter wvw = new WordVectorWriter(outFile, true,false,true,1);

        config.setConfigurationRule(WVTConfiguration.STEP_OUTPUT, new WVTConfigurationFact(wvw));

        config.setConfigurationRule(WVTConfiguration.STEP_VECTOR_CREATION, new WVTConfigurationFact(new TFIDF()));
        System.out.println(list);
        wvt.createVectors(list, config, wordList);

        wvw.close();
        outFile.close();
    }

    public static void createLibSVM(String path) throws IOException {
        CharSource source = Files.asCharSource(new File(path + "/wv.txt"), Charsets.UTF_8);
        CharSink sink = Files.asCharSink(new File(path + "/libsvm.txt"), Charsets.UTF_8);
        ImmutableList<String> lines = source.readLines();
        ArrayList<String> newlines = Lists.newArrayList();
        for(String l : lines){
            String[] l1 = l.split(";");
            String[] l2 = l1[1].split(" ");
            String type = l2[l2.length-1];
            String newline = type+" ";

            for (int i = 0; i <l2.length-1 ; i++) {
                newline += l2[i]+" ";
            }
            if(newline.trim().length()>type.length()){
                newlines.add(newline);
            }
        }
        sink.writeLines(newlines);
    }

}
